# -*- coding: utf-8 -*-

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from lxml import etree
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import time
from PIL import Image
import numpy as np
import os
import xlrd
import csv
from selenium.webdriver.common.keys import Keys

"""
WebDriver 只能在一个页面上对元素识别与定位，对于frame/iframe 表单内嵌页面上的元素无法直接定位
需要通过switch_to.frame()方法将当前定位的主体切换为frame/iframe 表单的内嵌页面中
#切换到表单
driver.switch_to.frame("x-URS-iframe")
打开新的标签页
js=f'window.open("{url}");'
driver.execute_script(js)
------Select------
# 选中这个标签，然后使用Select创建对象
selectTag = Select(driver.find_element_by_name("jumpMenu"))
# 根据索引选择
selectTag.select_by_index(1)
# 根据值选择
selectTag.select_by_value("值")
# 根据可视的文本选择
selectTag.select_by_visible_text("显示值")
# 取消选中所有选项
selectTag.deselect_all()
------Select------
"""

# 初始化
def init_driver(url):
    chrome_options = Options()
    # chrome_options.add_argument('--headless') # 无头模式
    chrome_options.add_argument('--disable-gpu')
    chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) # 设置开发模式，避免被检测
    # driver = webdriver.Chrome(chrome_options=chrome_options) # 已经设置PATH环境变量
    # 未在PATH环境变量中设置，需要设置驱动可执行路径
    driver = webdriver.Chrome(executable_path='D:\\webdriver\\chromedriver.exe', chrome_options=chrome_options)
    driver.maximize_window()  # 窗口最大化
    driver.get(url)
    driver.maximize_window()
    # html = driver.page_source
    # print(html)
    return driver


# 获取页面url
def get_url(drive, keyword, name, file):
    # 设置等待
    wait = WebDriverWait(driver, 20)

    # element = WebDriverWait(driver, 30).until(
    #         EC.presence_of_element_located((By.ID, "txt_1_value1")))
    # element.clear()
    # element.send_keys(keyword)
    # element = WebDriverWait(driver, 30).until(
    #     EC.presence_of_element_located((By.ID, "au_1_value1")))
    # element.clear()
    # element.send_keys(name)
    element = driver.find_element_by_id("article_1_value_1")
    element.clear()
    element.send_keys(keyword)
    element = driver.find_element_by_id("author_1_value_1")
    element.clear()
    element.send_keys(name)

    time.sleep(1)
    driver.find_element_by_id('author_1_value_2').click()
    driver.find_element_by_id('advSearchBtn').click()

    if len(driver.find_elements_by_xpath('div[@class="verifycode"]/div/input[@id="vericode"]')) > 0:
        while True:
            driver.delete_all_cookies()
            driver.refresh()
            element = driver.find_element_by_id("article_1_value_1")
            element.clear()
            element.send_keys(keyword)
            element = driver.find_element_by_id("author_1_value_1")
            element.clear()
            element.send_keys(name)
            driver.find_element_by_id('advSearchBtn').click()
            time.sleep(5)
            if len(driver.find_elements_by_xpath('div[@class="verifycode"]/div/input[@id="vericode"]')) > 0:
                pass
    while True:
        try:
            WebDriverWait(driver, 60).until(
                EC.presence_of_element_located((By.XPATH, '//p[@class="indexBar"]/b[1]')))
            break
        except:
            pass

    time.sleep(2)
    selector = etree.HTML(driver.page_source)
    xpath = selector.xpath('//table[@class="elist"]/tbody/tr')
    count = selector.xpath('//p[@class="indexBar"]/b[1]/text()')[0]
    window_height = driver.get_window_size()['height']  # 窗口高度
    page_height = driver.execute_script('return document.documentElement.scrollHeight')  # 页面高度
    area = window_height * 1/2
    driver.execute_script(f'document.documentElement.scrollTop={area};')
    driver.save_screenshot(file + ".png")
    if int(count) > 0:
        # screenshot(driver, window_height, page_height , file)
        ta = 0
        for x in xpath:
            title = "".join(x.xpath('./td[3]//text()')).replace("\n", "").strip()
            zz = "".join(x.xpath('./td[4]//text()')).replace("\n", "").strip()
            ta += 1
            if keyword == title:
                tar = '//table[@class="elist"]/tbody/tr[{tr}]/td[7]/lable/a'.format(tr=ta)
                #点击下一页
                driver.find_element_by_xpath(tar).click()
                # time.sleep(3)
                WebDriverWait(driver, 50).until(
                    EC.presence_of_element_located((By.XPATH, "//html")))
                now_handle = driver.current_window_handle
                # 获取所有窗口句柄
                all_handles = driver.window_handles
                for handle in all_handles:
                    if handle != now_handle:
                        driver.switch_to.window(handle)  # 切换新打开窗口
                        # window_height = driver.get_window_size()['height']  # 窗口高度
                        # page_height = driver.execute_script('return document.documentElement.scrollHeight')  # 页面高度
                        i = 1
                        while True:
                            time.sleep(3)
                            driver.execute_script(f'document.documentElement.scrollTop=0;')
                            area = window_height * 1/8
                            driver.execute_script(f'document.documentElement.scrollTop={area};')
                            driver.save_screenshot(file + "-" + str(i) + ".png")
                            # screenshot(driver, window_height, page_height , file + "-" + str(i))
                            i += 1
                            # 点击下一页
                            flag = True
                            for a in driver.find_elements_by_xpath('//div[@class="pageBar"]//a'):
                                if a.text == "下一页":
                                    a.click()
                                    flag = False
                                    break
                            if flag:
                                break
                                # 点击下一页
                        driver.close()
                # 返回主窗口
                driver.switch_to.window(now_handle)
                break
    driver.delete_all_cookies()
    driver.refresh()

# 获取页面url
def get_url1(drive, keyword, name, file):
    # 设置等待
    wait = WebDriverWait(driver, 20)

    element = driver.find_element_by_id("article_1_value_1")
    element.clear()
    element.send_keys(keyword)
    element = driver.find_element_by_id("author_1_value_1")
    element.clear()
    element.send_keys(name[0])
    time.sleep(1)
    driver.find_element_by_id('author_1_value_2').click()
    driver.find_element_by_id('advSearchBtn').click()

    if len(driver.find_elements_by_xpath('div[@class="verifycode"]/div/input[@id="vericode"]')) > 0:
        while True:
            driver.delete_all_cookies()
            driver.refresh()
            element = driver.find_element_by_id("article_1_value_1")
            element.clear()
            element.send_keys(keyword)
            element = driver.find_element_by_id("author_1_value_1")
            element.clear()
            element.send_keys(name)
            driver.find_element_by_id('advSearchBtn').click()
            time.sleep(5)
            if len(driver.find_elements_by_xpath('div[@class="verifycode"]/div/input[@id="vericode"]')) > 0:
                pass
    while True:
        try:
            WebDriverWait(driver, 60).until(
                EC.presence_of_element_located((By.XPATH, '//p[@class="indexBar"]/b[1]')))
            break
        except:
            pass

    time.sleep(2)
    selector = etree.HTML(driver.page_source)
    xpath = selector.xpath('//table[@class="elist"]/tbody/tr')
    count = selector.xpath('//p[@class="indexBar"]/b[1]/text()')[0]
    count1 = 0
    if int(count) > 0:
        ta = 0
        for x in xpath:
            title = "".join(x.xpath('./td[3]//text()')).replace("\n", "").strip()
            zz = "".join(x.xpath('./td[4]//text()')).replace("\n", "").strip()
            ta += 1
            if keyword == title:
                tar = '//table[@class="elist"]/tbody/tr[{tr}]/td[7]/lable/a'.format(tr=ta)
                #点击下一页
                driver.find_element_by_xpath(tar).click()
                WebDriverWait(driver, 50).until(
                    EC.presence_of_element_located((By.XPATH, "//html")))
                now_handle = driver.current_window_handle
                # 获取所有窗口句柄
                all_handles = driver.window_handles
                for handle in all_handles:
                    if handle != now_handle:
                        driver.switch_to.window(handle)  # 切换新打开窗口
                        while True:
                            time.sleep(2)
                            root = etree.HTML(driver.page_source)
                            selector1 = root.xpath('//table[@class="refDetail"]/tbody/tr')
                            for i in selector1:
                                flag = False
                                if len(i.xpath('./td[2]/label[1]/text()')) > 0:
                                    zz1 = i.xpath('./td[2]/label[1]/text()')[0].replace(".", "")
                                else:
                                    # zz1 = i.xpath('./td[2]/label[2]/a/text()')[0].replace(".", "")
                                    zz1 = "".join(i.xpath('./td[2]/label[2]//text()')).replace(".", "")
                                for ii in name:
                                    if ii in zz1:
                                        flag = True
                                        break
                                if flag != True:
                                    count1 += 1
                            # 点击下一页
                            flag = True
                            for a in driver.find_elements_by_xpath('//div[@class="pageBar"]//a'):
                                if a.text == "下一页":
                                    a.click()
                                    flag = False
                                    break
                            if flag:
                                break
                                # 点击下一页
                        driver.close()
                # 返回主窗口
                driver.switch_to.window(now_handle)
                break
    driver.delete_all_cookies()
    driver.refresh()
    return count1


def screenshot(driver, window_height, page_height, filename):
    # 截图，截取长截图
    # from PIL import Image
    # import numpy as np
    # import os
    # window_height = driver.get_window_size()['height']  # 窗口高度
    # page_height = driver.execute_script('return document.documentElement.scrollHeight')  # 页面高度
    if page_height > window_height:
        n = page_height // window_height  # 需要滚动的次数
        driver.save_screenshot('tmppic.png')
        base_mat = np.atleast_2d(Image.open('tmppic.png'))  # 打开截图并转为二维矩阵
        for i in range(n):
            driver.execute_script(f'document.documentElement.scrollTop={window_height*(i+1)};')
            time.sleep(.5)
            driver.save_screenshot(f'tmppic_{i}.png')  # 保存截图
            if i == n - 1:
                img = Image.open(f'tmppic_{i}.png')
                x1 = 0
                y1 = page_height - (page_height % window_height)
                x2 = img.width
                y2 = page_height - (page_height % window_height)
                img2 = img.crop((1, y1, x2, y2)) # 剪切图片
                img2 = img.crop([0,270,x2,img.size[1]*3/4]) # 剪切图片
                img2.save(f'tmppic_{i}.png') # 保存图片，覆盖完整的网页图片
            mat = np.atleast_2d(Image.open(f'tmppic_{i}.png'))  # 打开截图并转为二维矩阵
            base_mat = np.append(base_mat, mat, axis=0)  # 拼接图片的二维矩阵
        Image.fromarray(base_mat).save(filename + ".png")
        for file in os.listdir('../SeleniumDemo/'):
            if "tmppic" in file:
                os.remove(file)
    else:
        driver.save_screenshot(filename + ".png")
    # 截图，截取长截图

def short_sc(el,b):
    sc_hight=614
    start_higth = el.location["y"]
    js = "scrollTo(0,%s)" % (start_higth)
    b.execute_script(js) # 执行js
    time.sleep(0.5)
    fp = r"test.png" # 图片地址，运行的话，改一下
    b.get_screenshot_as_file(fp)
    img = Image.open(fp=fp)
    img2 = img.crop((el.location["x"], 0, el.size["width"] + el.location["x"], el.size["height"])) # 剪切图片
    img2.save(fp)
def long_sc(el,b):
    sc_hight=614
    count = int(el.size["height"] / sc_hight) # 元素的高度除以你每次截多少就是次数
    start_higth = el.location["y"] # 元素的初始高度
    max_px = start_higth + (count - 1) * sc_hight # for循环中最大的px
    last_px = el.size["height"] + start_higth - sc_hight # 元素最底部的位置
    surplus_px = last_px - max_px # 剩余的边的高度
    img_path = [] # 用来存放图片地址
    for i in range(0, count):
        js = "scrollTo(0,%s)" % (start_higth + i * sc_hight) # 用于移动滑轮，每次移动614px，初始值是元素的初始高度
        b.execute_script(js) # 执行js
        time.sleep(0.5)
        fp = r"%s.png" % i # 图片地址，运行的话，改一下
        b.get_screenshot_as_file(fp) # 屏幕截图,这里是截取是完整的网页图片，你可以打断点看一下图片
        img = Image.open(fp=fp)
        img2 = img.crop((el.location["x"], 0, el.size["width"] + el.location["x"], sc_hight)) # 剪切图片
        img2.save(fp) # 保存图片，覆盖完整的网页图片
        img_path.append(fp) # 添加图片路径
        time.sleep(0.5)
        print(js)
    else:
        js = "scrollTo(0,%s)" % last_px # 滚动到最后一个位置
        b.execute_script(js)
        fp = r"last.png"
        b.get_screenshot_as_file(fp)
        img = Image.open(fp=fp)
        print((el.location["x"], sc_hight - surplus_px, el.size["width"] + el.location["x"], sc_hight))
        img2 = img.crop((el.location["x"], sc_hight - surplus_px, el.size["width"] + el.location["x"], sc_hight))
        img2.save(fp)
        img_path.append(fp)
        print(js)
    new_img = Image.new("RGB", (el.size["width"], el.size["height"])) # 创建一个新图片,大小为元素的大小
    k = 0
    for i in img_path:
        tem_img = Image.open(i)
        new_img.paste(tem_img, (0, sc_hight * k)) # 把图片贴上去,间隔一个截图的距离
        k += 1
    else:
        new_img.save(r"test.png") # 保存

    b.maximize_window()#最大化窗口
    # b.get_screenshot_as_file(fp)
    sc_hight=614#你屏幕截图默认的大小，可以去截一张，去画图里面看看是多少像素，我这里是614像素
    # b.switch_to.frame(b.find_element_by_xpath('//*[@id="intro"]/iframe'))
    el=b.find_element_by_id("Form1")#找到元素
    # el=b.find_element_by_class_name("botbox")#找到元素
    if el.size["height"]>sc_hight:
        long_sc(el,b)
    else:
        short_sc(el,b)


def close_driver(driver):
    driver.quit()



if __name__ == '__main__':
    url = "http://ref.cnki.net/REF/AdvSearch"
    driver = init_driver(url)
    data = xlrd.open_workbook("keywords.xlsx")
    table = data.sheets()[0]
    title_list = []
    name_list = []
    name_list1 = []
    xh = []
    qk = []
    cb = []
    zz = []
    for rownum in range(table.nrows):
        if rownum > 0:
            title_list.append(table.row_values(rownum)[1].strip())
            name_list.append(table.row_values(rownum)[2].strip())
            xh.append(table.row_values(rownum)[0])
            qk.append(table.row_values(rownum)[3])
            cb.append(table.row_values(rownum)[4])
            cb.append(table.row_values(rownum)[3])
    for x,y in enumerate(title_list):
        # name = name_list[x].split(',')[0]
        name_list1 = []
        for i in name_list[x].split(','):
            name_list1.append(i)
        print("=======================")
        print(title_list[x])
        # 截图
        # get_url(driver, title_list[x], name_list1[0], str(x + 1))
        # 截图
        # 取数据
        with open("count.csv", 'a+', encoding="utf-8-sig", newline='') as f:
            count = get_url1(driver, title_list[x], name_list1, str(x + 1))
            print("================")
            print(count)
            data = {
                "a1":xh[x],
                "a2":title_list[x],
                "a3":name_list[x],
                "a4":qk[x],
                "a5":cb[x],
                "a6":count,
            }
            csv_writer = csv.writer(f)
            csv_writer.writerow([xh[x], title_list[x], name_list[x], qk[x], cb[x], count])
        # 取数据